#!/bin/bash
#Written by Matthew J Vona
#This program email diagnostic data to the site operator and supervisor
trap cleanup 5 6 7 8 9 
cleanup()
{
	echo "Cleaning up.."
	killall sjinn 2> /dev/null;
	echo "Done"
	exit;
}

while read line; do
eval $line
done < Config/config
clean_errors()
{
for (( i = 0 ; i < 10 ; i++ ))
        do
		echo "" > Data/errors0$i
 	done
for (( i = 10 ; i < 25 ; i++ ))
        do
		echo "" > Data/errors$i
 	done

}
#########################################################
#THIS FUNCTION WRITTEN TO HANDLE MODEM DIAL OUTS	#							 
#--MJV							#						 
#########################################################
while :
do
		#############################	
		#LOOP CONTROL/TIME VARIABLES#
		#############################        
		yesterday=`date -d yesterday "+%d"`
		month=`date -d yesterday "+%m"`
		hour=`date "+%H"`
		minute=`date "+%M"`
		ingested="0"
		reported="0"
		tar_flag="0"


		#################################################################################
		#	MAKE A RAW DATA TAR FILE ONCE PER DAY					#
		#################################################################################
		if [ $hour == "00" ] && [ $minute == "01" ]
		then
		    tar_flag=0
		fi
		
		
		#################################################################################
		#	INGEST AND SEND DMS FILES @ MINUTE 13 EACH HOUR				#
		#################################################################################
		if [ $minute == "05" ]
		then
			if [ "$ingested" = "0" ]; then
        			{
        			#updated to run ingest only once with both hours--MJV
        			ingest_day=`date -d "2 hours ago" "+%d"`
        			ingest_month=`date -d "2 hours ago" "+%m"`
        			ingest_year=`date -d "2 hours ago" "+%Y"` 
        			ingest_hour=`date -d "2 hours ago" "+%H"` 
        			new_ingest_day=`date "+%d"`
        			new_ingest_month=`date "+%m"`
        			new_ingest_year=`date "+%Y"` 
        			new_ingest_hour=`date "+%H"`
        				#IF THE INGEST_HOUR TOKEN IS > NEW INGEST HOUR TOKEN
        				#THEN RUN INGEST TWICE, ONCE FOR EACH HOUR. OTHERWISE
        				#RUN IT ONCE WITH REGEX
        				if [ $ingest_hour -gt $new_ingest_hour ]; then
        					{
        					/bin/bash sys_resources/ingest.sh $ingest_hour $ingest_day $ingest_month $ingest_year \
						1> /dev/null 2> /dev/null
        					wait $!;
        					/bin/bash sys_resources/ingest.sh $new_ingest_hour $new_ingest_day $new_ingest_month $new_ingest_year \
						1> /dev/null 2> /dev/null
        					sleep 10;
        					}
        				else
        					{
						/bin/bash sys_resources/ingest.sh \[${ingest_hour},${new_ingest_hour}\] $ingest_day $ingest_month\
						 $ingest_year 1> /dev/null 2> /dev/null   #REGEX     				
        					}
        				fi
        		
        			sleep 10;   
        			extract_day=`date -d "3 hour ago" "+%d"`
        			extract_month=`date -d "3 hour ago" "+%m"`
        			extract_year=`date -d "3 hour ago" "+%Y"` 
        			extract_hour=`date -d "3 hour ago" "+%H"`
        			bash sys_resources/dmsout.sh "$extract_year-$extract_month-$extract_day $extract_hour:00:00"\
				 "$new_ingest_year-$new_ingest_month-$new_ingest_day $new_ingest_hour:00:00" 1>/dev/null 2>/dev/null 
        			#Call to backpoll program
        			bash sys_resources/backpoll.sh &> /dev/null 
        			#/bin/bash sys_resources/spmout.sh $ingest_hour $ingest_day $ingest_month $ingest_year
        			wait;
        			sleep 10;
				if (ls [0-9]*.MIN &> /dev/null); then
				 {
				 count=0
					######################################################################
					#CODE TO LOOK FOR AN IP ADDRESS AND IF NOT, GET ONE VIA MODEM        #
					######################################################################
					ip_present=`/sbin/ifconfig | grep -v "127.0.0.1" | grep inet\ addr | wc -l`
					if [ "$ip_present" == "0" ]; then
					    {
						      wvdial 1> Data/CARBLogger.log&
						      sleep 60;
							 modem_connected=1	
						}
					fi
					for i in `ls -t [0-9]*.MIN`
						do
						{
							if [ $count -lt 6 ]; then
								bash sys_resources/carbput.sh $i 1>/dev/null 2>/dev/null
								sleep 10
								count=`expr $count + 1`
							fi
						}
						done
						#KILL THE MODEM
						if [ "$modem_connected" == "1" ]; then
					    		{	
								killall wvdial 1> /dev/null 1> /dev/null
							}
						fi
			
			
				 }
				fi
				ingested=1
        		#####CLEANING UP PROCESSING FILES#######
        		rm -f .[0-9]*prcssng
        		}
 			fi
		#DEBUG		
		echo "REPORTING LOOP" 		
		sleep 60;
		fi


		
		#################################################################################
		#	COMPILE AND SEND THE EMAILS 2X PER DAY					#
		#################################################################################
		if [ $hour == $hour1 ] || [ $hour == $hour2 ] && [ "$reported" == "0" ] && [ "$minute" == "10" ] ; then
		       {
			rm -f sys_resources/.tmpcom &>/dev/null;
			echo "" >> sys_resources/.tmpcom
			echo "Site: $sitnam CARBLogger 	Reporting for `date "+%m"`/`date "+%d"`" >> sys_resources/.tmpcom
			echo "Site Operator:${siteop} Site Supervisor:${sitesup} " >> sys_resources/.tmpcom
			number=$(ps -fe | grep testd | grep -v grep | wc -l)
			echo "Number of Active Instruments: $number                               " >> sys_resources/.tmpcom
			echo "DATA STATE - - - - - - - - - - - - - - - - - - " >> sys_resources/.tmpcom
			size=$(ls -lah Data | grep total)
			al_cons=$(cat Data/errors[0-2][0-9] 2>/dev/null | grep : | wc -l)
			echo "Data Directory Size: ${size} 	Alerts Recorded: $al_cons                        #" >> sys_resources/.tmpcom
			echo " -----------------------ALERT DETAILS- - - - - - - - - - - - - - - - - - - " >> sys_resources/.tmpcom
			count_drv=1
			for i in ${active_inst};
				do
			   		 drv_list[$count_drv]="$i"
					 usr_count=$(ls -B resources/$i/.usr*)
					 for k in $usr_count
					 	do
					     temp=$(echo "${k#*/*/.usr}")
					   	 chn_nam[$count_drv]=$(cat resources/$i/.usr$temp | grep -o "\-r [-A-Za-z0-9_]*" | awk '{ print $2 }')
				 	     count_drv=`expr $count_drv + 1`
				 	     max_count_drv="$count_drv"
					 	done
			     	done
			#TEST TO SEE IF ANY ERRORS FILES EXIST; IF SO PREPARE THEM       	
			if (stat Data/errors* 1> /dev/null 2>/dev/null); then
				{
					#read all errors files, sor the contents to count the numbers for each
					cat Data/errors[0-2]* | grep -v ^$ | sort >Data/.tmpstats
					#create an errors.txt file from the sorted contents
					cp Data/.tmpstats Data/errors.txt              
					#for each instrument tally the number of errors present
					#and write in the .tmpcom file in the sys_resources folder 
					  for (( i = 1 ; i < max_count_drv ; i++ ))
					      do
						      num_err=$(cat Data/.tmpstats | grep "${chn_nam[$i]} "| wc -l)
						      echo "${chn_nam[$i]} has $num_err errors" >> sys_resources/.tmpcom                       
					      done
				}
			fi
			#SET THE EMAIL SUBJECT BASED ON THE NUMBER OF ERRORS		       	
			subject="CARBLogger: Status"
		       	if [ $al_cons -gt 0 ]; then
		       		{
		       		subject=$subject" $sitnam Includes Errors"
		       		}
		       		else
				{
		      		 subject=$subject" $sitnam Nominal"
				}
		       	fi

			############ADDED FUNCTION FOR SENDING VIA MODEM/MSMTP/SLOW CONNECTIONS
			############CHECK FOR AN INTERNET CONNECTION#############################
			#If we have a non-loopback address, we can go ahead and send out the email.
			#If not, we will check to see if wvdial is installed, and if so make a call to it.
			#Regardless, we will send the email, which gets stored for later transmission
			#by Mutt if no IP address was obtained. Finally if we did get an IP address,
			#we will set a flag to terminate wvdial after we send out. FOR PRODUCTION
			#PURPOSES WE SET THE IP_PRESNT TO 1, ASSUMING THAT IP IS PRESENT. TO DEPLOY A MODEM
			#UNCOMMENT THE IP_PRESENT LINE.
			#########################################################################
			#ip_present='1'
			ip_present=`/sbin/ifconfig | grep -v "127.0.0.1" | grep inet\ addr | wc -l`
			if [ "$ip_present" == "0" ]; then
				{
			#########################################################################
			# NO INTERNET PRESENT							#
			#########################################################################
					wvdial 1> Data/CARBLogger.log&
					sleep 60;
					recips="$siteop $sitesup";  
					if [ $scnd_email == "y" ]; then
						{
						recips="$recips $sitescnd";
						}
					fi
					#################################################################################
					# CREATE A LIST OF BAM CONFIG CHANGE FILES TO ATTACH				#
					#################################################################################	
					listcfgs=''
					for i in `ls -lah resources/*BAM1020*_drv/.bamcfg* 2> /dev/null | awk '{ print $9 }'`
					do
						listcfgs="-a $i ${listcfgs}"
					done
	
					cat "sys_resources/.tmpcom" | mutt -s "$subject" -a "Data/errors.txt" $listcfgs -- $recips && sleep 10;
			    		rm "Data/errors.txt"
			    		rm resources/*BAM1020*_drv/.bamcfg* 2>/dev/null
			    		sleep 10;
					if (stat text_summary_[0-3][0-9]* 1>/dev/null 2>/dev/null); then
					{
						echo "Data Extracts" | mutt -s "$sitnam data" -a text_summary_[0-3][0-9]* -a raw[0-9][0-9]* -- $recips && sleep 30;
						rm -f text_summary_[0-3][0-9]* 1> /dev/null 2> /dev/null
						rm -f raw[0-9][0-9]* 1> /dev/null 2> /dev/null
					}
					fi
					now=`date`
					echo "CARBLogger Time before NIST SYNC: $now" >> "sys_resources/.tmpcom"; 
					#grab a working ntp server
					ntp_srvr=`ntpq -p -n | grep -v === | grep -v remote | awk '($8 < 3000){ print $1 }'`
					#find the path to ntpdate
					#sync the carblogger to ntp server
					for i in $ntp_srvr;
					do
						{
							/usr/sbin/ntpdate -s -p 8 -u $i &> /dev/null ;  
						}
					done
				now=`date`
				echo "CARBLogger Time after NIST SYNC: $now" >> "sys_resources/.tmpcom"; 
				cat "sys_resources/.tmpcom" | mutt -s "$subject" -a "Data/errors.txt" -- $recips && sleep 60;
				rm "Data/errors.txt"
				#IF THE SYSTEM IS GENERATING DATA EXTRACTS, MAIL THOSE OUT
		
				if (stat text_summary_[0-3][0-9]* 1> /dev/null); then
				{
					echo "Data Extracts" | mutt -s "$sitnam data" -a text_summary_[0-3][0-9]* -a raw[0-9][0-9]* -- $recips && sleep 10

				}
				fi
		
				clean_errors
		
				}
			else 
			#########################################################################
			# INTERNET IS PRESENT							#
			#########################################################################
			{
				recips="$siteop $sitesup";  
				if [ $scnd_email == "y" ]; then
						{
						recips="$recips $sitescnd";
						}
				fi 
				#################################################################################
				# CREATE A LIST OF BAM CONFIG CHANGE FILES TO ATTACH				#
				#################################################################################	
				listcfgs=''
				for i in `ls -lah resources/*BAM1020*_drv/.bamcfg* 2> /dev/null | awk '{ print $9 }'`
				do
					listcfgs="-a $i ${listcfgs}"
			
				done
	
				cat "sys_resources/.tmpcom" | mutt -s "$subject" -a "Data/errors.txt" $listcfgs -- $recips && sleep 10;
			    	rm "Data/errors.txt"
			    	rm resources/*BAM1020*_drv/.bamcfg* 2>/dev/null
			    	sleep 10
					if (stat text_summary_[0-3][0-9]* 1>/dev/null 2>/dev/null); then
					{
						echo "Data Extracts" | mutt -s "$sitnam data" -a text_summary_[0-3][0-9]* -a raw[0-9][0-9]* -- $recips\
						 && sleep 30;
						sleep 60
						rm -f text_summary_[0-3][0-9]* 1> /dev/null 2> /dev/null
						rm -f raw[0-9][0-9]* 1> /dev/null 2> /dev/null
					}
					fi
				clean_errors;
				sleep 60
				reported=1
				}
			fi
			clean_errors
			sleep 10;
			#If the conditions were present causing wvdial to spawn we kill wvdial now
			#HERE WE INDICATE THAT A REPORT WAS SENT OUT THIS HOUR
		 		#killall wvdial 1> /dev/null 2> /dev/null       
		       }
		#elif [ $hour != $hour1 ] && [ $hour != $hour2 ] ; then
	#	{
#			#SINCE THIS IS NOT A REPORTING HOUR, WE RESET THE REPORTING FLAG HERE#
	#		reported=0
	#		if [ "$hour" -eq "03" ]; then
	#			{
	#			tar czf `date -d "yesterday" "+raw%d%m%Y"`.tgz Data/`date -d "yesterday" "+${saroad}-%Y-%d-%m"`
	#			bash sys_resources/spmout.sh $yesterday $month `date -d yesterday "+%Y"` 1> /dev/null 2> /dev/null	
	#			}
	#		fi
	#	
	#		#CODE FIXED TO NOT REMOVE DATA PRIOR TO INGEST AND OR SUBMITTAL
	#		if [ $tar_flag -eq 0 ] && [ `date "+%d"` -eq "10" ]; then
	#	
	##				{
	#					rm Data/*`date -d "3 months ago" "+%m"`
	#					tar_flag=1
	#				}
	#			fi
	#		}
	#		fi
		#echo "IDLE LOOP"
		sleep 15
		#sleep 3600;
		#}	
	else
	{
		
		#SINCE THIS IS NOT A REPORTING HOUR, WE RESET THE REPORTING FLAG HERE
			reported=0
			if [ "$hour" -eq "03" ]; then
				{
				tar czf `date -d "yesterday" "+raw%d%m%Y"`.tgz Data/`date -d "yesterday" "+${saroad}-%Y-%d-%m"`
				bash sys_resources/spmout.sh $yesterday $month `date -d yesterday "+%Y"` 1> /dev/null 2> /dev/null	
				}
			fi
		
			#CODE FIXED TO NOT REMOVE DATA PRIOR TO INGEST AND OR SUBMITTAL
			if [ $tar_flag -eq 0 ] && [ `date "+%d"` -eq "10" ]; then
		
			{	if (tar czf `date -d "3 months ago" "+%m-%Y"`.tgz Data/*`date -d "3 months ago" "+%m"` 1> /dev/null); then
					{
						rm Data/*`date -d "3 months ago" "+%m"`
						tar_flag=1
					}
				fi
			}
			fi		

		reported==0
		ingested=0		
		
		#echo "OUTSIDE IDLE LOOP"
		sleep 15

	}
	fi
done;

